# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
from scrapy.utils.project import get_project_settings
import pymysql
import pymongo

class KuaidailiPipeline(object):
    # 重写构造方法，在这打开文件
    def __init__(self):
        # 文件的打开写到这里，仅会执行一次
        self.fp = open('kuaidaili.txt', 'w', encoding='utf-8')
        
    def open_spider(self, spider):
        pass

    # 在这里处理每一个item
    def process_item(self, item, spider):
        # 将这个对象转化为字典
        obj = dict(item)
        # print(obj)
        # 将obj转化为字符串 然后写入到文件
        string = json.dumps(obj, ensure_ascii=False)
        self.fp.write(string + '\n')
        return item

    # 重写这个方法，在关闭spider的时候将文件资源关闭
    def close_spider(self, spider):
        self.fp.close()
# 存入到mysql中
class MysqlPipeline(object):
    """docstring for MysqlPipeline"""
    def __init__(self):
        settings = get_project_settings()
        self.host = settings['DB_HOST']
        self.port = settings['DB_PORT']
        self.user = settings['DB_USER']
        self.pwd = settings['DB_PWD']
        self.name = settings['DB_NAME']
        self.charset = settings['DB_CHARSET']
        # 链接数据库
        self.connect()

    def connect(self):
        self.conn = pymysql.connect(host=self.host,
                             port=self.port,
                             user=self.user,
                             password=self.pwd,
                             db=self.name,
                             charset=self.charset)
        self.cursor = self.conn.cursor()

    def close_spider(self, spider):
        self.conn.close()
        self.cursor.close()

    def process_item(self, item, spider):
        sql = 'insert into ip_list(ip, port) values("%s", "%s")' % (item['ip'], item['port'])
        # 执行sql语句
        self.cursor.execute(sql)
        # 提交之后才会生效
        self.conn.commit()
        return item

class MongoDBPipeline(object):
    """docstring for MysqlPipeline"""
    def __init__(self):
        settings = get_project_settings()
        self.host = settings['M_DB_HOST']
        self.port = settings['M_DB_PORT']
        self.name = settings['M_DB_NAME']
        # 链接数据库
        self.connect()

    def connect(self):
        self.mongocli = pymongo.MongoClient(host = self.host, port = self.port)
        self.dbname = mongocli[self.name]
        self.sheetname = self.dbname["ip_list"]

    def close_spider(self, spider):
        self.mongocli.close()

    def process_item(self, item, spider):
        data = json.loads(item)
        # 将数据插入到sheetname表里
        self.sheetname.insert(data)
        return item